######################################################################
# Expression data subclass for the Data object
# (to sit inside <EP::Common::DBI::Dataset>)
######################################################################
package EP::Core::Dataset::Data::exp;

use strict;
use base 'EP::Core::Dataset::Data';

use EP::Config;

use PDL ();
use PDL::IO::FlexRaw;
use PDL::Graphics::PGPLOT;
use PGPLOT;
use PDL::NiceSlice;

use EP::Common::General;
use EP::Common::DBI::Dataset_Metadata;
use EP::Common::DBI::Dataset_Metadata_Type;
use EP::Core::Data::Parser::exp::user;

use EP::Visualization::Grid;
use EP::Visualization::Hooks;
use EP::Visualization::AnnotationGraphic;
use EP::Visualization::LineGraphic;
use EP::Visualization::HeatmapGraphic;

use File::Temp;
use File::Basename;
use Tie::IxHash;

use EP::Rclient;

sub process_metadata
    {
    my $self = shift;

    $self->_read_raw_data() if not exists $self->{matrix};

    my $dataset_folder_path = $EP::Config::h->{EPNG_USRDATA_PATH} . "/" . $self->{owner}->folder->location;
    my $dataset_file = $self->{owner}->filename . ".bin";
    my $legend_image = $self->{owner}->filename . "_legend.png";
    my $histogram_image = $self->{owner}->filename . "_histogram.png";
    my $lineplot_image = $self->{owner}->filename . "_lineplot.png";
    my $heatmap_image = $self->{owner}->filename . "_heatmap.png";
    my $heatmap_scale_image = $self->{owner}->filename . "_heatmap_scale.png";

    my ($columns, $rows )                 = $self->{matrix}->dims();
    my ($mean, $rms, $median, $min, $max) = EP::Rclient->R(<<__RCODE2__);
app_root = "$EP::Config::h->{ROOT}";
source("$EP::Config::h->{ROOT}/cgi-lib/tmpl/R/ep.expmeta.tmpl.R");
dataset = ep.readBin("$dataset_folder_path/$dataset_file");
ep.exp.getstats(dataset);
__RCODE2__

    my $rowscale = 10;
    $rowscale = 1 if $rows > 2000;

    EP::Rclient->R(<<__RCODE__);
app_root = "$EP::Config::h->{ROOT}";
source("$EP::Config::h->{ROOT}/cgi-lib/tmpl/R/ep.expmeta.tmpl.R");
dataset = ep.readBin("$dataset_folder_path/$dataset_file");
ep.vis.exp.legend( dataset, "$dataset_folder_path/$legend_image" );
ep.vis.exp.histogram( dataset, "$dataset_folder_path/$histogram_image" );
ep.vis.exp.lineplot( dataset, "$dataset_folder_path/$lineplot_image" );
ep.vis.exp.heatmap( dataset, "$dataset_folder_path/$heatmap_image", "$dataset_folder_path/$heatmap_scale_image", xscale = 10, yscale = $rowscale );
__RCODE__

    # old histogram dwawing facility (replacing with R)
    if (0)
        {
        my $png = dev ( "$dataset_folder_path/$histogram_image/PNG", { Aspect => 1, WindowWidth => 3 } );

        # if the whole matrix is all repeats of one value just draw a line at that value
        if ( $self->{matrix}->uniq->dim(0) == 1 )
            {
            line ( PDL::long ( $self->{matrix}->at ( 0,0 ), $self->{matrix}->at ( 0,0 ) ) );
            }
        else
            {
            bin ( PDL::hist ( $self->{matrix} ), { COLOUR => 'RED' } );
            }

        close_window ( $png );
        }

#   my ( $lineplot_image, $lineplot_width, $lineplot_height ) = $self->_create_lineplot_visualization();
#   my ( $heatmap_image, $heatmap_width, $heatmap_height )    = $self->_create_heatmap_visualization();
#   my ($mean, $rms, $median, $min, $max) = @{$self->stats()};


    my @factors;
    foreach my $factor ( $self->{column}->{(keys %{$self->{column}})[0]}->{full_annot}->Keys )
        {
        my $fv = new Tie::IxHash;

        # obtain unique factor values
        foreach ( 0 .. $columns - 1 )
            {
            next if not exists $self->{column}->{$_};
            my $this_fv = $self->{column}->{$_}->{full_annot}->Values ( $self->{column}->{$_}->{full_annot}->Indices ( $factor ) );
            $fv->Push ( $this_fv, 1 );
            }

        push @factors, "$factor: " . scalar $fv->Keys . " groups";
        }

    $self->update_processed_metadata(
        { mean  => $mean
        , stdev => $rms
        , median => $median
        , min => $min
        , max => $max
        , num_columns => $columns
        , num_rows => $rows
        , factors => join ( " ; ", @factors )
        , lineplot => $lineplot_image
        , heatmap  => $heatmap_image
        , heatmap_scale => $heatmap_scale_image
        , histogram => $histogram_image
        , legend => $legend_image
        });

    return
        { heatmap =>
            { type => "image/png"
            , label => "Dataset heatmap"
            , value => $EP::Config::h->{EPNG_USRDATA_URL} . "/" . $self->{owner}->folder->location . "/" . $heatmap_image
            }
        , lineplot =>
            { type => "image/png"
            , label => "Dataset lineplot"
            , value => $EP::Config::h->{EPNG_USRDATA_URL} . "/" . $self->{owner}->folder->location . "/" . $lineplot_image
            }
        };
    }

# sub: matrix
#   See <raw_data>.
sub matrix {
  my $self = shift;
  return $self->raw_data ( @_ );
}

# sub: raw_data
#   Returns the dataset matrix pdl
sub raw_data {
  my $self = shift;
  return $self->{matrix} if exists $self->{matrix};
  return $self->{matrix} if $self->_read_raw_data ();
  die "Failed to read data!";
}

# sub: rows
#   Returns the rows hash for the dataset
sub rows {
  my $self = shift;
  return $self->{row} if exists $self->{row};
  return $self->{row} if $self->_read_raw_data ();
  die "Failed to read data!";
}

# sub: cols
#   Returns the columns hash for the dataset
sub cols {
  my $self = shift;
  return $self->{column} if exists $self->{column};
  return $self->{column} if $self->_read_raw_data ();
  die "Failed to read data!";
}

sub stats {
  my $self = shift;
  $self->{stats} = [$self->{matrix}->stats()] if not exists $self->{stats};

  return $self->{stats};
}

# sub: _read_raw_data
# reads the dataset matrix pdl and row/col annotations
sub _read_raw_data {
  my $self = shift;
  local ( *COLUMNS, *ROWS );

  my ($folder) = $EP::Config::h->{EPNG_USRDATA_PATH} . "/" . $self->{owner}->folder->location;
  my ($filename) = $self->{owner}->filename;

  #read the row identifiers and annotations
  open (ROWS, "$folder/$filename.rows")
    or die "Couldn't open $filename.rows, $!";

  chomp (my $row_annot_header = <ROWS>);
  my @row_annot_names = split "\t", $row_annot_header;
  shift @row_annot_names;	# shift because the first item there is "ID"

  while (<ROWS>) {
    chomp $_;
    my ($row_id, @row_annot) = split /\t/, $_;

    my $row_annotation = new Tie::IxHash;
    $row_annotation->Push ( $row_annot_names[$_] => $row_annot[$_] ) foreach 0 .. $#row_annot_names;

    $self->{row}{$row_id} = { header => $row_id,
			      mappped_row_header => "",
			      annot => \@row_annot,
			      full_annot => $row_annotation
			    };
  }
  close ROWS;

  #read the column identifiers and annotations
  open (COLUMNS, "$folder/$filename.columns")
    or die "Couldn't open $filename.columns, $!";

  chomp (my $col_annot_header = <COLUMNS>);
  my @col_annot_names = split "\t", $col_annot_header;
  shift @col_annot_names;	# shift because the first item there is "ID"

  while (<COLUMNS>) {
    chomp $_; 
    my ($col_id, @col_annot) = split /\t/, $_;

    my $col_annotation = new Tie::IxHash;
    $col_annotation->Push ( $col_annot_names[$_]  => $col_annot[$_] ) foreach 0 .. $#col_annot_names;

    $self->{column}{$col_id} = { header => $col_id,
				 annot => \@col_annot,
				 full_annot => $col_annotation
			       };
  }
  close COLUMNS;

  my ($rows, $cols);
  ( $rows, $cols, $self->{matrix} ) = readflex ( "$folder/$filename.bin" );

  my @dims = $self->{matrix}->dims();
  die "Problem with reading in PDL matrix from $folder/$filename.bin" if $rows != $dims[1] or $cols != $dims[0];

  warn "Read " . $self->{matrix}->info() if DEBUG;
  $self->{matrix}->inplace->setnantobad;
  $self->{matrix}->badflag(1);

  #add some extra stats to object
  $self->stats();
}

# sub: _create_lineplot_visualization
# produces a lineplot visualization for the whole dataset
sub _create_lineplot_visualization {
  my $self = shift;

  my $grid = new EP::Visualization::Grid ( 2, 2 );

  my $heatmap_data    = $self->raw_data;
  my @heatmap_V_hooks = sort { $a <=> $b} keys %{$self->{row}};
  my @heatmap_H_hooks = sort { $a <=> $b} keys %{$self->{column}};

  my $V_hook = new EP::Visualization::Hooks ( \@heatmap_V_hooks, $self->{row} );
  my $H_hook = new EP::Visualization::Hooks ( \@heatmap_H_hooks, $self->{column} );

  my ($cell_height, $cell_width ) = ( 1, 1 );

  if ( @heatmap_V_hooks < 200 ) {
    $cell_height = 12;
  } elsif ( @heatmap_V_hooks < 500 ) {
    $cell_height = 6;
  }

  if ( @heatmap_H_hooks < 200 ) {
    $cell_width = 12;
  } elsif ( @heatmap_H_hooks < 500 ) {
    $cell_width = 6;
  }

  my $col_annotate = $self->{column};
  my @col_annots;
  push @col_annots, $col_annotate->{$_} foreach sort { $a <=> $b} keys %{$col_annotate};

  my $col_annotation = new EP::Visualization::AnnotationGraphic ( data => \@col_annots, 'font-size' => $cell_width, 'annot-width' => 50 );
  $col_annotation->setHook ( $H_hook );
  $col_annotation->setOrientation(-90);

  my $cluster_lineplot = new EP::Visualization::LineGraphic ( data => $heatmap_data, plot_height => 150, cell_width => $cell_width );
  $cluster_lineplot->setVerticalHookSet   ( $V_hook );
  $cluster_lineplot->setHorizontalHookSet ( $H_hook );

  $grid->fillCellArray ( [[ undef, $col_annotation   ],
			  [ undef, $cluster_lineplot ]] );

  die "Failed to generate visualization!" if not defined $grid;

  my $svg_output = new File::Temp ( DIR => $EP::Config::h->{EPNG_USRDATA_PATH} . "/" . $self->{owner}->folder->location,
				    SUFFIX => ".svg",
				    UNLINK => 0 );

  my $png_output = new File::Temp ( DIR => $EP::Config::h->{EPNG_USRDATA_PATH} . "/" . $self->{owner}->folder->location,
				    SUFFIX => ".png",
				    UNLINK => 0 );

#  print $svg_output $grid->to_svg ( zoom => 0, width => $cell_width * scalar @heatmap_H_hooks + 50, height => 335 );
  print $svg_output $grid->to_png ( pngout => $png_output );

  return ( basename ( $png_output ),  $grid->getGridWidth(), $grid->getGridHeight() );
}

# sub: _create_heatmap_visualization
# produces a heatmap visualization for the whole dataset
sub _create_heatmap_visualization {
  my $self = shift;

  my $grid = new EP::Visualization::Grid ( 2, 2 );

  my $heatmap_data    = $self->raw_data;
  my @heatmap_V_hooks = sort { $a <=> $b} keys %{$self->{row}};
  my @heatmap_H_hooks = sort { $a <=> $b} keys %{$self->{column}};

  my $V_hook = new EP::Visualization::Hooks ( \@heatmap_V_hooks, $self->{row} );
  my $H_hook = new EP::Visualization::Hooks ( \@heatmap_H_hooks, $self->{column} );

  my ($cell_height, $cell_width ) = ( 1, 1 );

  if ( @heatmap_V_hooks < 200 ) {
    $cell_height = 12;
  } elsif ( @heatmap_V_hooks < 500 ) {
    $cell_height = 6;
  }

  if ( @heatmap_H_hooks < 200 ) {
    $cell_width = 12;
  } elsif ( @heatmap_H_hooks < 500 ) {
    $cell_width = 6;
  }

  my $col_annotate = $self->{column};
  my @col_annots;
  push @col_annots, $col_annotate->{$_} foreach sort { $a <=> $b} keys %{$col_annotate};

  my $row_annotate = $self->{row};
  my @row_annots;
  push @row_annots, $row_annotate->{$_} foreach sort { $a <=> $b} keys %{$row_annotate};

  my $col_annotation = new EP::Visualization::AnnotationGraphic ( data => \@col_annots, 'font-size' => $cell_width, 'annot-width' => 50 );
  $col_annotation->setHook ( $H_hook );
  $col_annotation->setOrientation(-90);

  my $row_annotation = new EP::Visualization::AnnotationGraphic ( data => \@row_annots, 'font-size' => $cell_height, 'annot-width' => 50, 'no-color-efs' => 1 );
  $row_annotation->setHook ( $V_hook );

  my $heatmap = new EP::Visualization::HeatmapGraphic ( data => $heatmap_data, cell_height => $cell_height, cell_width => $cell_width );
  $heatmap->setVerticalHookSet   ( $V_hook );
  $heatmap->setHorizontalHookSet ( $H_hook );

  $grid->fillCellArray ( [[ undef,           $col_annotation   ],
			  [ $row_annotation, $heatmap          ]] );

  die "Failed to generate visualization!" if not defined $grid;

  my $svg_output = new File::Temp ( DIR => $EP::Config::h->{EPNG_USRDATA_PATH} . "/" . $self->{owner}->folder->location,
				    SUFFIX => ".svg",
				    UNLINK => 0 );

  my $png_output = new File::Temp ( DIR => $EP::Config::h->{EPNG_USRDATA_PATH} . "/" . $self->{owner}->folder->location,
				    SUFFIX => ".png",
				    UNLINK => 0 );

#  print $svg_output $grid->to_svg ( zoom => 0 );
  $grid->to_png ( pngout => $png_output ) if $cell_height > 1;
  $grid->to_png ( pngout => $png_output, width => 400, height => 600 ) if $cell_height == 1;

  return ( basename ( $png_output ),  $grid->getGridWidth(), $grid->getGridHeight() );
}

# sub: get_experimental_factors
#   returns an indexed hash of experimental factors and their values
sub get_experimental_factors {
  my $self = shift;

  my $columns = $self->{matrix}->dim(0);

  my $factors = new Tie::IxHash;
  foreach my $factor ( $self->{column}->{0}->{full_annot}->Keys ) {
    my $fv = new Tie::IxHash;

    # obtain unique factor values
    foreach my $column ( 0 .. $columns - 1 ) {
      my $this_fv = $self->{column}->{$column}->{full_annot}->Values ( $self->{column}->{$column}->{full_annot}->Indices ( $factor ) );
      $fv->Push ( $this_fv, [ @{$fv->Pop ( $this_fv )}, $column ] );
    }

    $factors->Push ( $factor, $fv );
  }

  return $factors;
}

# sub: add_experimental_factor
#  adds a new experimental factor to the data [matrix].
# params:
#  factor - name of factor to add,
#  fvs    - ref to array of new values. must be as many as columns in the matrix.
sub add_experimental_factor {
  my $self = shift;
  my ( $factor, $ref_fvs ) = @_;

  $self->_read_raw_data() if not exists $self->{matrix};

  my @fvs = @$ref_fvs;
  my $columns = $self->{matrix}->dim(0);

  die "Not enough factor values supplied, " . scalar @fvs . ", for new factor $factor, $columns needed!" if $columns != @fvs;

  warn "Adding " . scalar @fvs . " values for factor $factor" if DEBUG;

  foreach my $column ( 0 .. $columns - 1 ) {
    $self->{column}->{$column}->{full_annot}->Push ( $factor, shift @fvs );
  }

  EP::Core::Data::Parser::exp::user->_write_dataset_data ( $self->{owner}, $self );
  $self->process_metadata ();

  return "OK";
}

# sub: delete_experimental_factor
#   deletes a factor
# params:
#   factor - name of factor to delete
sub delete_experimental_factor {
  my $self = shift;
  my ( $factor ) = @_;

  $self->_read_raw_data() if not exists $self->{matrix};
  die "No such factor, $factor!" if not $self->{column}->{0}->{full_annot}->Indices ( $factor );

  my $columns = $self->{matrix}->dim(0);

  warn "Deleting $factor..." if DEBUG;

  foreach my $column ( 0 .. $columns - 1 ) {
    $self->{column}->{$column}->{full_annot}->Delete ( $factor );
  }

  EP::Core::Data::Parser::exp::user->_write_dataset_data ( $self->{owner}, $self );
  $self->process_metadata ();

  return "OK";
}

# sub: identify_annotations
#   (re-)identifies row annotation columns
# params:
#   identannots - array ref of new annot identifiers to replace the existing ones
sub identify_annotations {
  my $self = shift;
  my ( $identannots ) = @_;
  my @idas = @{$identannots};

  $self->_read_raw_data() if not exists $self->{matrix};
  die "Not enough annotation identifiers supplied, " . scalar @idas . "!" if @idas != scalar $self->{row}->{0}->{full_annot}->Keys;

  warn "Resetting " . scalar @idas . " annotations" if DEBUG;

  my $rows = $self->{matrix}->dim(1);
  foreach my $row ( 0 .. $rows - 1 ) {
    foreach my $annot ( $self->{row}->{$row}->{full_annot}->Keys ) {
      my $idx = $self->{row}->{$row}->{full_annot}->Indices ( $annot );
      my $annot_val = $self->{row}->{$row}->{full_annot}->Values ( $idx );
      $self->{row}->{$row}->{full_annot}->Replace ( $idx, $annot_val, $idas[$idx] );
    }
  }

  EP::Core::Data::Parser::exp::user->_write_dataset_data ( $self->{owner}, $self );
  $self->process_metadata ();

  return "OK";
}

1;
